{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_op = pd.read_csv('operation_feature.csv')\n",
    "df_trans = pd.read_csv('transction_feature.csv')\n",
    "\n",
    "df_op_train = df_op[df_op.Tag != -1].reset_index(drop=True)\n",
    "df_op_test = df_op[df_op.Tag == -1].reset_index(drop=True)\n",
    "\n",
    "df_trans_train = df_trans[df_trans.Tag != -1].reset_index(drop=True)\n",
    "df_trans_test = df_trans[df_trans.Tag == -1].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29727, 101) (30523, 188)\n"
     ]
    }
   ],
   "source": [
    "print(df_op_train.shape, df_trans_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object `pd.fillna` not found.\n"
     ]
    }
   ],
   "source": [
    "?pd.fillna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.merge(df_trans_train, df_op_train, on=['UID', 'Tag'], how='outer')\n",
    "df_train = df_train.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(31486, 101) (30593, 188)\n"
     ]
    }
   ],
   "source": [
    "print(df_op_test.shape, df_trans_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_test = pd.merge(df_op_test, df_trans_test, on=['UID', 'Tag'], how='outer')\n",
    "df_test.drop('Tag',axis=1, inplace=True)\n",
    "df_test = df_test.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('tran_data.csv', index=False)\n",
    "df_test.to_csv('test_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = list(set(set(df_op_test.UID.unique()) | set(df_trans_test.UID.unique())))\n",
    "sub_csv = pd.DataFrame({'UID':cols, 'Tag':[1 for i in range(len(cols))]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_csv.to_csv('./sub_csv.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def blending_model(res, weights, model_name, subs):\n",
    "    \"\"\"\n",
    "    结果加权平均\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    blend_model = subs\n",
    "    value = []\n",
    "\n",
    "    def weight_average(df):\n",
    "        val = df[0:].tolist()\n",
    "        v = np.average(val, weights=weights)\n",
    "        value.append(v)\n",
    "\n",
    "    res.T.apply(weight_average)\n",
    "    blend_model[tag_hd.Tag] = value\n",
    "    m = sum(weights) / len(weights)\n",
    "    print(m)\n",
    "    blend_model.to_csv(sub_base_path + 'blending_' + model_name + str(m) + '_.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(31588, 286)\n",
      "(31179, 286)\n",
      "(31588, 286)\n",
      "0\n",
      "Training until validation scores don't improve for 30 rounds.\n",
      "[10]\tvalid_0's binary_logloss: 0.536362\tvalid_1's binary_logloss: 0.537313\n",
      "[20]\tvalid_0's binary_logloss: 0.43601\tvalid_1's binary_logloss: 0.438608\n",
      "[30]\tvalid_0's binary_logloss: 0.367158\tvalid_1's binary_logloss: 0.372067\n",
      "[40]\tvalid_0's binary_logloss: 0.3172\tvalid_1's binary_logloss: 0.324519\n",
      "[50]\tvalid_0's binary_logloss: 0.279848\tvalid_1's binary_logloss: 0.289144\n",
      "[60]\tvalid_0's binary_logloss: 0.251576\tvalid_1's binary_logloss: 0.262832\n",
      "[70]\tvalid_0's binary_logloss: 0.229487\tvalid_1's binary_logloss: 0.242799\n",
      "[80]\tvalid_0's binary_logloss: 0.212169\tvalid_1's binary_logloss: 0.227349\n",
      "[90]\tvalid_0's binary_logloss: 0.197592\tvalid_1's binary_logloss: 0.215041\n",
      "[100]\tvalid_0's binary_logloss: 0.185222\tvalid_1's binary_logloss: 0.204689\n",
      "[110]\tvalid_0's binary_logloss: 0.174579\tvalid_1's binary_logloss: 0.195858\n",
      "[120]\tvalid_0's binary_logloss: 0.165878\tvalid_1's binary_logloss: 0.189012\n",
      "[130]\tvalid_0's binary_logloss: 0.157986\tvalid_1's binary_logloss: 0.182805\n",
      "[140]\tvalid_0's binary_logloss: 0.150792\tvalid_1's binary_logloss: 0.177669\n",
      "[150]\tvalid_0's binary_logloss: 0.14465\tvalid_1's binary_logloss: 0.173427\n",
      "[160]\tvalid_0's binary_logloss: 0.13908\tvalid_1's binary_logloss: 0.169626\n",
      "[170]\tvalid_0's binary_logloss: 0.133735\tvalid_1's binary_logloss: 0.165694\n",
      "[180]\tvalid_0's binary_logloss: 0.129103\tvalid_1's binary_logloss: 0.162808\n",
      "[190]\tvalid_0's binary_logloss: 0.124481\tvalid_1's binary_logloss: 0.159783\n",
      "[200]\tvalid_0's binary_logloss: 0.120328\tvalid_1's binary_logloss: 0.156981\n",
      "[210]\tvalid_0's binary_logloss: 0.116375\tvalid_1's binary_logloss: 0.154504\n",
      "[220]\tvalid_0's binary_logloss: 0.112633\tvalid_1's binary_logloss: 0.152305\n",
      "[230]\tvalid_0's binary_logloss: 0.109159\tvalid_1's binary_logloss: 0.150168\n",
      "[240]\tvalid_0's binary_logloss: 0.10574\tvalid_1's binary_logloss: 0.148197\n",
      "[250]\tvalid_0's binary_logloss: 0.102624\tvalid_1's binary_logloss: 0.146401\n",
      "[260]\tvalid_0's binary_logloss: 0.0996609\tvalid_1's binary_logloss: 0.144775\n",
      "[270]\tvalid_0's binary_logloss: 0.0968605\tvalid_1's binary_logloss: 0.143291\n",
      "[280]\tvalid_0's binary_logloss: 0.0942225\tvalid_1's binary_logloss: 0.141932\n",
      "[290]\tvalid_0's binary_logloss: 0.0915455\tvalid_1's binary_logloss: 0.140439\n",
      "[300]\tvalid_0's binary_logloss: 0.0890515\tvalid_1's binary_logloss: 0.139119\n",
      "[310]\tvalid_0's binary_logloss: 0.0868221\tvalid_1's binary_logloss: 0.137978\n",
      "[320]\tvalid_0's binary_logloss: 0.0845614\tvalid_1's binary_logloss: 0.136738\n",
      "[330]\tvalid_0's binary_logloss: 0.0825608\tvalid_1's binary_logloss: 0.135737\n",
      "[340]\tvalid_0's binary_logloss: 0.0805773\tvalid_1's binary_logloss: 0.134703\n",
      "[350]\tvalid_0's binary_logloss: 0.0785977\tvalid_1's binary_logloss: 0.133565\n",
      "[360]\tvalid_0's binary_logloss: 0.0767883\tvalid_1's binary_logloss: 0.132718\n",
      "[370]\tvalid_0's binary_logloss: 0.075038\tvalid_1's binary_logloss: 0.131824\n",
      "[380]\tvalid_0's binary_logloss: 0.0735\tvalid_1's binary_logloss: 0.131123\n",
      "[390]\tvalid_0's binary_logloss: 0.0719596\tvalid_1's binary_logloss: 0.130334\n",
      "[400]\tvalid_0's binary_logloss: 0.0704987\tvalid_1's binary_logloss: 0.129692\n",
      "[410]\tvalid_0's binary_logloss: 0.0692562\tvalid_1's binary_logloss: 0.12906\n",
      "[420]\tvalid_0's binary_logloss: 0.0680197\tvalid_1's binary_logloss: 0.128528\n",
      "[430]\tvalid_0's binary_logloss: 0.0668707\tvalid_1's binary_logloss: 0.127953\n",
      "[440]\tvalid_0's binary_logloss: 0.0657312\tvalid_1's binary_logloss: 0.127439\n",
      "[450]\tvalid_0's binary_logloss: 0.0647122\tvalid_1's binary_logloss: 0.126996\n",
      "[460]\tvalid_0's binary_logloss: 0.0635807\tvalid_1's binary_logloss: 0.126424\n",
      "[470]\tvalid_0's binary_logloss: 0.0626728\tvalid_1's binary_logloss: 0.126071\n",
      "[480]\tvalid_0's binary_logloss: 0.0617624\tvalid_1's binary_logloss: 0.125753\n",
      "[490]\tvalid_0's binary_logloss: 0.0610387\tvalid_1's binary_logloss: 0.125447\n",
      "[500]\tvalid_0's binary_logloss: 0.0602009\tvalid_1's binary_logloss: 0.125171\n",
      "[510]\tvalid_0's binary_logloss: 0.0594654\tvalid_1's binary_logloss: 0.124856\n",
      "[520]\tvalid_0's binary_logloss: 0.0587979\tvalid_1's binary_logloss: 0.124562\n",
      "[530]\tvalid_0's binary_logloss: 0.0582754\tvalid_1's binary_logloss: 0.124393\n",
      "[540]\tvalid_0's binary_logloss: 0.0577058\tvalid_1's binary_logloss: 0.124171\n",
      "[550]\tvalid_0's binary_logloss: 0.0572022\tvalid_1's binary_logloss: 0.124102\n",
      "[560]\tvalid_0's binary_logloss: 0.0566392\tvalid_1's binary_logloss: 0.123819\n",
      "[570]\tvalid_0's binary_logloss: 0.0562726\tvalid_1's binary_logloss: 0.123752\n",
      "[580]\tvalid_0's binary_logloss: 0.0557716\tvalid_1's binary_logloss: 0.123594\n",
      "[590]\tvalid_0's binary_logloss: 0.0553218\tvalid_1's binary_logloss: 0.123442\n",
      "[600]\tvalid_0's binary_logloss: 0.0549947\tvalid_1's binary_logloss: 0.123354\n",
      "[610]\tvalid_0's binary_logloss: 0.0546816\tvalid_1's binary_logloss: 0.123292\n",
      "[620]\tvalid_0's binary_logloss: 0.0542896\tvalid_1's binary_logloss: 0.123164\n",
      "[630]\tvalid_0's binary_logloss: 0.0540034\tvalid_1's binary_logloss: 0.123041\n",
      "[640]\tvalid_0's binary_logloss: 0.0537308\tvalid_1's binary_logloss: 0.122973\n",
      "[650]\tvalid_0's binary_logloss: 0.0534814\tvalid_1's binary_logloss: 0.12293\n",
      "[660]\tvalid_0's binary_logloss: 0.0532008\tvalid_1's binary_logloss: 0.122856\n",
      "[670]\tvalid_0's binary_logloss: 0.052889\tvalid_1's binary_logloss: 0.122696\n",
      "[680]\tvalid_0's binary_logloss: 0.0527027\tvalid_1's binary_logloss: 0.122613\n",
      "[690]\tvalid_0's binary_logloss: 0.0525396\tvalid_1's binary_logloss: 0.122545\n",
      "[700]\tvalid_0's binary_logloss: 0.0523881\tvalid_1's binary_logloss: 0.122504\n",
      "[710]\tvalid_0's binary_logloss: 0.0521832\tvalid_1's binary_logloss: 0.122375\n",
      "[720]\tvalid_0's binary_logloss: 0.0519891\tvalid_1's binary_logloss: 0.122325\n",
      "[730]\tvalid_0's binary_logloss: 0.0518795\tvalid_1's binary_logloss: 0.122299\n",
      "[740]\tvalid_0's binary_logloss: 0.0517424\tvalid_1's binary_logloss: 0.122242\n",
      "[750]\tvalid_0's binary_logloss: 0.0516592\tvalid_1's binary_logloss: 0.122244\n",
      "[760]\tvalid_0's binary_logloss: 0.0514555\tvalid_1's binary_logloss: 0.122178\n",
      "[770]\tvalid_0's binary_logloss: 0.0513324\tvalid_1's binary_logloss: 0.122152\n",
      "[780]\tvalid_0's binary_logloss: 0.0511508\tvalid_1's binary_logloss: 0.122084\n",
      "[790]\tvalid_0's binary_logloss: 0.0510777\tvalid_1's binary_logloss: 0.122073\n",
      "[800]\tvalid_0's binary_logloss: 0.0509369\tvalid_1's binary_logloss: 0.122046\n",
      "[810]\tvalid_0's binary_logloss: 0.050893\tvalid_1's binary_logloss: 0.122077\n",
      "[820]\tvalid_0's binary_logloss: 0.050809\tvalid_1's binary_logloss: 0.122067\n",
      "[830]\tvalid_0's binary_logloss: 0.0506961\tvalid_1's binary_logloss: 0.122068\n",
      "[840]\tvalid_0's binary_logloss: 0.0505259\tvalid_1's binary_logloss: 0.12201\n",
      "[850]\tvalid_0's binary_logloss: 0.0503825\tvalid_1's binary_logloss: 0.121989\n",
      "[860]\tvalid_0's binary_logloss: 0.0502729\tvalid_1's binary_logloss: 0.121931\n",
      "[870]\tvalid_0's binary_logloss: 0.0501486\tvalid_1's binary_logloss: 0.121889\n",
      "[880]\tvalid_0's binary_logloss: 0.0500954\tvalid_1's binary_logloss: 0.121895\n",
      "[890]\tvalid_0's binary_logloss: 0.0499871\tvalid_1's binary_logloss: 0.121869\n",
      "[900]\tvalid_0's binary_logloss: 0.049866\tvalid_1's binary_logloss: 0.121814\n",
      "[910]\tvalid_0's binary_logloss: 0.0497556\tvalid_1's binary_logloss: 0.121744\n",
      "[920]\tvalid_0's binary_logloss: 0.0496456\tvalid_1's binary_logloss: 0.121716\n",
      "[930]\tvalid_0's binary_logloss: 0.049629\tvalid_1's binary_logloss: 0.121722\n",
      "[940]\tvalid_0's binary_logloss: 0.0495334\tvalid_1's binary_logloss: 0.121715\n",
      "[950]\tvalid_0's binary_logloss: 0.0494543\tvalid_1's binary_logloss: 0.121698\n",
      "[960]\tvalid_0's binary_logloss: 0.0493986\tvalid_1's binary_logloss: 0.121698\n",
      "[970]\tvalid_0's binary_logloss: 0.0493111\tvalid_1's binary_logloss: 0.121682\n",
      "[980]\tvalid_0's binary_logloss: 0.0492293\tvalid_1's binary_logloss: 0.121647\n",
      "[990]\tvalid_0's binary_logloss: 0.0490926\tvalid_1's binary_logloss: 0.121578\n",
      "[1000]\tvalid_0's binary_logloss: 0.0489984\tvalid_1's binary_logloss: 0.121564\n",
      "[1010]\tvalid_0's binary_logloss: 0.0489729\tvalid_1's binary_logloss: 0.121565\n",
      "[1020]\tvalid_0's binary_logloss: 0.0488997\tvalid_1's binary_logloss: 0.121544\n",
      "[1030]\tvalid_0's binary_logloss: 0.0488433\tvalid_1's binary_logloss: 0.121537\n",
      "[1040]\tvalid_0's binary_logloss: 0.0487833\tvalid_1's binary_logloss: 0.121508\n",
      "[1050]\tvalid_0's binary_logloss: 0.0487243\tvalid_1's binary_logloss: 0.121512\n",
      "[1060]\tvalid_0's binary_logloss: 0.0486877\tvalid_1's binary_logloss: 0.121501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1070]\tvalid_0's binary_logloss: 0.0486877\tvalid_1's binary_logloss: 0.121501\n",
      "[1080]\tvalid_0's binary_logloss: 0.0486877\tvalid_1's binary_logloss: 0.121501\n",
      "Early stopping, best iteration is:\n",
      "[1057]\tvalid_0's binary_logloss: 0.0486877\tvalid_1's binary_logloss: 0.121501\n",
      "0.738389731621937\n",
      "1\n",
      "Training until validation scores don't improve for 30 rounds.\n",
      "[10]\tvalid_0's binary_logloss: 0.538366\tvalid_1's binary_logloss: 0.54072\n",
      "[20]\tvalid_0's binary_logloss: 0.437714\tvalid_1's binary_logloss: 0.44202\n",
      "[30]\tvalid_0's binary_logloss: 0.368551\tvalid_1's binary_logloss: 0.374695\n",
      "[40]\tvalid_0's binary_logloss: 0.318092\tvalid_1's binary_logloss: 0.325888\n",
      "[50]\tvalid_0's binary_logloss: 0.280375\tvalid_1's binary_logloss: 0.290627\n",
      "[60]\tvalid_0's binary_logloss: 0.251143\tvalid_1's binary_logloss: 0.264016\n",
      "[70]\tvalid_0's binary_logloss: 0.22856\tvalid_1's binary_logloss: 0.243728\n",
      "[80]\tvalid_0's binary_logloss: 0.210598\tvalid_1's binary_logloss: 0.227879\n",
      "[90]\tvalid_0's binary_logloss: 0.19567\tvalid_1's binary_logloss: 0.215319\n",
      "[100]\tvalid_0's binary_logloss: 0.183209\tvalid_1's binary_logloss: 0.205101\n",
      "[110]\tvalid_0's binary_logloss: 0.172731\tvalid_1's binary_logloss: 0.197142\n",
      "[120]\tvalid_0's binary_logloss: 0.163664\tvalid_1's binary_logloss: 0.190128\n",
      "[130]\tvalid_0's binary_logloss: 0.155775\tvalid_1's binary_logloss: 0.184534\n",
      "[140]\tvalid_0's binary_logloss: 0.148712\tvalid_1's binary_logloss: 0.179698\n",
      "[150]\tvalid_0's binary_logloss: 0.142383\tvalid_1's binary_logloss: 0.175509\n",
      "[160]\tvalid_0's binary_logloss: 0.136841\tvalid_1's binary_logloss: 0.171861\n",
      "[170]\tvalid_0's binary_logloss: 0.131547\tvalid_1's binary_logloss: 0.168297\n",
      "[180]\tvalid_0's binary_logloss: 0.126694\tvalid_1's binary_logloss: 0.165227\n",
      "[190]\tvalid_0's binary_logloss: 0.121978\tvalid_1's binary_logloss: 0.162463\n",
      "[200]\tvalid_0's binary_logloss: 0.118007\tvalid_1's binary_logloss: 0.160011\n",
      "[210]\tvalid_0's binary_logloss: 0.11401\tvalid_1's binary_logloss: 0.157612\n",
      "[220]\tvalid_0's binary_logloss: 0.110381\tvalid_1's binary_logloss: 0.155698\n",
      "[230]\tvalid_0's binary_logloss: 0.106881\tvalid_1's binary_logloss: 0.153649\n",
      "[240]\tvalid_0's binary_logloss: 0.10372\tvalid_1's binary_logloss: 0.151688\n",
      "[250]\tvalid_0's binary_logloss: 0.100546\tvalid_1's binary_logloss: 0.149914\n",
      "[260]\tvalid_0's binary_logloss: 0.0976396\tvalid_1's binary_logloss: 0.14808\n",
      "[270]\tvalid_0's binary_logloss: 0.0948732\tvalid_1's binary_logloss: 0.146558\n",
      "[280]\tvalid_0's binary_logloss: 0.0920531\tvalid_1's binary_logloss: 0.145025\n",
      "[290]\tvalid_0's binary_logloss: 0.0895278\tvalid_1's binary_logloss: 0.143592\n",
      "[300]\tvalid_0's binary_logloss: 0.0870923\tvalid_1's binary_logloss: 0.142315\n",
      "[310]\tvalid_0's binary_logloss: 0.0848779\tvalid_1's binary_logloss: 0.14116\n",
      "[320]\tvalid_0's binary_logloss: 0.0828259\tvalid_1's binary_logloss: 0.140191\n",
      "[330]\tvalid_0's binary_logloss: 0.0807313\tvalid_1's binary_logloss: 0.138979\n",
      "[340]\tvalid_0's binary_logloss: 0.078709\tvalid_1's binary_logloss: 0.137936\n",
      "[350]\tvalid_0's binary_logloss: 0.0768983\tvalid_1's binary_logloss: 0.137036\n",
      "[360]\tvalid_0's binary_logloss: 0.0751025\tvalid_1's binary_logloss: 0.136147\n",
      "[370]\tvalid_0's binary_logloss: 0.0735029\tvalid_1's binary_logloss: 0.135378\n",
      "[380]\tvalid_0's binary_logloss: 0.0718917\tvalid_1's binary_logloss: 0.134647\n",
      "[390]\tvalid_0's binary_logloss: 0.070471\tvalid_1's binary_logloss: 0.133913\n",
      "[400]\tvalid_0's binary_logloss: 0.0690881\tvalid_1's binary_logloss: 0.133254\n",
      "[410]\tvalid_0's binary_logloss: 0.0676823\tvalid_1's binary_logloss: 0.132718\n",
      "[420]\tvalid_0's binary_logloss: 0.0664049\tvalid_1's binary_logloss: 0.13213\n",
      "[430]\tvalid_0's binary_logloss: 0.0652844\tvalid_1's binary_logloss: 0.131682\n",
      "[440]\tvalid_0's binary_logloss: 0.0642483\tvalid_1's binary_logloss: 0.131302\n",
      "[450]\tvalid_0's binary_logloss: 0.0632196\tvalid_1's binary_logloss: 0.130839\n",
      "[460]\tvalid_0's binary_logloss: 0.0622492\tvalid_1's binary_logloss: 0.130378\n",
      "[470]\tvalid_0's binary_logloss: 0.0613572\tvalid_1's binary_logloss: 0.130023\n",
      "[480]\tvalid_0's binary_logloss: 0.0605085\tvalid_1's binary_logloss: 0.129638\n",
      "[490]\tvalid_0's binary_logloss: 0.059662\tvalid_1's binary_logloss: 0.129241\n",
      "[500]\tvalid_0's binary_logloss: 0.0589461\tvalid_1's binary_logloss: 0.128945\n",
      "[510]\tvalid_0's binary_logloss: 0.0582477\tvalid_1's binary_logloss: 0.128549\n",
      "[520]\tvalid_0's binary_logloss: 0.0576522\tvalid_1's binary_logloss: 0.128308\n",
      "[530]\tvalid_0's binary_logloss: 0.0570886\tvalid_1's binary_logloss: 0.128097\n",
      "[540]\tvalid_0's binary_logloss: 0.0565497\tvalid_1's binary_logloss: 0.127842\n",
      "[550]\tvalid_0's binary_logloss: 0.05601\tvalid_1's binary_logloss: 0.127596\n",
      "[560]\tvalid_0's binary_logloss: 0.0555227\tvalid_1's binary_logloss: 0.127432\n",
      "[570]\tvalid_0's binary_logloss: 0.0551463\tvalid_1's binary_logloss: 0.127373\n",
      "[580]\tvalid_0's binary_logloss: 0.0547107\tvalid_1's binary_logloss: 0.127188\n",
      "[590]\tvalid_0's binary_logloss: 0.0543448\tvalid_1's binary_logloss: 0.127106\n",
      "[600]\tvalid_0's binary_logloss: 0.0540229\tvalid_1's binary_logloss: 0.127034\n",
      "[610]\tvalid_0's binary_logloss: 0.053682\tvalid_1's binary_logloss: 0.126951\n",
      "[620]\tvalid_0's binary_logloss: 0.0533202\tvalid_1's binary_logloss: 0.126828\n",
      "[630]\tvalid_0's binary_logloss: 0.0530457\tvalid_1's binary_logloss: 0.12676\n",
      "[640]\tvalid_0's binary_logloss: 0.0527393\tvalid_1's binary_logloss: 0.126629\n",
      "[650]\tvalid_0's binary_logloss: 0.0524739\tvalid_1's binary_logloss: 0.126523\n",
      "[660]\tvalid_0's binary_logloss: 0.0522807\tvalid_1's binary_logloss: 0.126493\n",
      "[670]\tvalid_0's binary_logloss: 0.0520182\tvalid_1's binary_logloss: 0.126401\n",
      "[680]\tvalid_0's binary_logloss: 0.051741\tvalid_1's binary_logloss: 0.126324\n",
      "[690]\tvalid_0's binary_logloss: 0.0515424\tvalid_1's binary_logloss: 0.126306\n",
      "[700]\tvalid_0's binary_logloss: 0.0512575\tvalid_1's binary_logloss: 0.126165\n",
      "[710]\tvalid_0's binary_logloss: 0.051046\tvalid_1's binary_logloss: 0.126084\n",
      "[720]\tvalid_0's binary_logloss: 0.0509173\tvalid_1's binary_logloss: 0.126097\n",
      "[730]\tvalid_0's binary_logloss: 0.0507519\tvalid_1's binary_logloss: 0.126001\n",
      "[740]\tvalid_0's binary_logloss: 0.0505809\tvalid_1's binary_logloss: 0.125951\n",
      "[750]\tvalid_0's binary_logloss: 0.0504492\tvalid_1's binary_logloss: 0.125927\n",
      "[760]\tvalid_0's binary_logloss: 0.0502422\tvalid_1's binary_logloss: 0.125799\n",
      "[770]\tvalid_0's binary_logloss: 0.050064\tvalid_1's binary_logloss: 0.125729\n",
      "[780]\tvalid_0's binary_logloss: 0.0499487\tvalid_1's binary_logloss: 0.125695\n",
      "[790]\tvalid_0's binary_logloss: 0.0498051\tvalid_1's binary_logloss: 0.125657\n",
      "[800]\tvalid_0's binary_logloss: 0.0497178\tvalid_1's binary_logloss: 0.125665\n",
      "[810]\tvalid_0's binary_logloss: 0.0495523\tvalid_1's binary_logloss: 0.125579\n",
      "[820]\tvalid_0's binary_logloss: 0.0494538\tvalid_1's binary_logloss: 0.125597\n",
      "[830]\tvalid_0's binary_logloss: 0.0493445\tvalid_1's binary_logloss: 0.125596\n",
      "[840]\tvalid_0's binary_logloss: 0.0492715\tvalid_1's binary_logloss: 0.12558\n",
      "[850]\tvalid_0's binary_logloss: 0.0491739\tvalid_1's binary_logloss: 0.125551\n",
      "[860]\tvalid_0's binary_logloss: 0.0490661\tvalid_1's binary_logloss: 0.125505\n",
      "[870]\tvalid_0's binary_logloss: 0.0489763\tvalid_1's binary_logloss: 0.125489\n",
      "[880]\tvalid_0's binary_logloss: 0.0489131\tvalid_1's binary_logloss: 0.125487\n",
      "[890]\tvalid_0's binary_logloss: 0.0488463\tvalid_1's binary_logloss: 0.125438\n",
      "[900]\tvalid_0's binary_logloss: 0.0487625\tvalid_1's binary_logloss: 0.125442\n",
      "[910]\tvalid_0's binary_logloss: 0.0486673\tvalid_1's binary_logloss: 0.12538\n",
      "[920]\tvalid_0's binary_logloss: 0.0485769\tvalid_1's binary_logloss: 0.125331\n",
      "[930]\tvalid_0's binary_logloss: 0.0484277\tvalid_1's binary_logloss: 0.125263\n",
      "[940]\tvalid_0's binary_logloss: 0.0483452\tvalid_1's binary_logloss: 0.125269\n",
      "[950]\tvalid_0's binary_logloss: 0.0482327\tvalid_1's binary_logloss: 0.125251\n",
      "[960]\tvalid_0's binary_logloss: 0.0481627\tvalid_1's binary_logloss: 0.125236\n",
      "[970]\tvalid_0's binary_logloss: 0.0480742\tvalid_1's binary_logloss: 0.125179\n",
      "[980]\tvalid_0's binary_logloss: 0.0479792\tvalid_1's binary_logloss: 0.12516\n",
      "[990]\tvalid_0's binary_logloss: 0.0478828\tvalid_1's binary_logloss: 0.12516\n",
      "[1000]\tvalid_0's binary_logloss: 0.0478411\tvalid_1's binary_logloss: 0.125157\n",
      "[1010]\tvalid_0's binary_logloss: 0.0477998\tvalid_1's binary_logloss: 0.125159\n",
      "Early stopping, best iteration is:\n",
      "[987]\tvalid_0's binary_logloss: 0.047907\tvalid_1's binary_logloss: 0.125143\n",
      "0.7409568261376898\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "Training until validation scores don't improve for 30 rounds.\n",
      "[10]\tvalid_0's binary_logloss: 0.537457\tvalid_1's binary_logloss: 0.54286\n",
      "[20]\tvalid_0's binary_logloss: 0.436406\tvalid_1's binary_logloss: 0.445447\n",
      "[30]\tvalid_0's binary_logloss: 0.36711\tvalid_1's binary_logloss: 0.379491\n",
      "[40]\tvalid_0's binary_logloss: 0.317754\tvalid_1's binary_logloss: 0.333442\n",
      "[50]\tvalid_0's binary_logloss: 0.281315\tvalid_1's binary_logloss: 0.299794\n",
      "[60]\tvalid_0's binary_logloss: 0.253\tvalid_1's binary_logloss: 0.273532\n",
      "[70]\tvalid_0's binary_logloss: 0.230352\tvalid_1's binary_logloss: 0.252712\n",
      "[80]\tvalid_0's binary_logloss: 0.212821\tvalid_1's binary_logloss: 0.23719\n",
      "[90]\tvalid_0's binary_logloss: 0.198297\tvalid_1's binary_logloss: 0.224129\n",
      "[100]\tvalid_0's binary_logloss: 0.186226\tvalid_1's binary_logloss: 0.213789\n",
      "[110]\tvalid_0's binary_logloss: 0.175824\tvalid_1's binary_logloss: 0.20526\n",
      "[120]\tvalid_0's binary_logloss: 0.166883\tvalid_1's binary_logloss: 0.198164\n",
      "[130]\tvalid_0's binary_logloss: 0.158826\tvalid_1's binary_logloss: 0.191892\n",
      "[140]\tvalid_0's binary_logloss: 0.151581\tvalid_1's binary_logloss: 0.186507\n",
      "[150]\tvalid_0's binary_logloss: 0.145139\tvalid_1's binary_logloss: 0.181757\n",
      "[160]\tvalid_0's binary_logloss: 0.139254\tvalid_1's binary_logloss: 0.177425\n",
      "[170]\tvalid_0's binary_logloss: 0.134104\tvalid_1's binary_logloss: 0.174193\n",
      "[180]\tvalid_0's binary_logloss: 0.129221\tvalid_1's binary_logloss: 0.170892\n",
      "[190]\tvalid_0's binary_logloss: 0.124792\tvalid_1's binary_logloss: 0.168108\n",
      "[200]\tvalid_0's binary_logloss: 0.120455\tvalid_1's binary_logloss: 0.165233\n",
      "[210]\tvalid_0's binary_logloss: 0.116544\tvalid_1's binary_logloss: 0.162748\n",
      "[220]\tvalid_0's binary_logloss: 0.11278\tvalid_1's binary_logloss: 0.160233\n",
      "[230]\tvalid_0's binary_logloss: 0.109217\tvalid_1's binary_logloss: 0.157959\n",
      "[240]\tvalid_0's binary_logloss: 0.105999\tvalid_1's binary_logloss: 0.155857\n",
      "[250]\tvalid_0's binary_logloss: 0.102963\tvalid_1's binary_logloss: 0.153919\n",
      "[260]\tvalid_0's binary_logloss: 0.0999127\tvalid_1's binary_logloss: 0.151975\n",
      "[270]\tvalid_0's binary_logloss: 0.0972554\tvalid_1's binary_logloss: 0.150424\n",
      "[280]\tvalid_0's binary_logloss: 0.0945273\tvalid_1's binary_logloss: 0.148834\n",
      "[290]\tvalid_0's binary_logloss: 0.0917955\tvalid_1's binary_logloss: 0.147109\n",
      "[300]\tvalid_0's binary_logloss: 0.0893741\tvalid_1's binary_logloss: 0.145773\n",
      "[310]\tvalid_0's binary_logloss: 0.0871223\tvalid_1's binary_logloss: 0.144641\n",
      "[320]\tvalid_0's binary_logloss: 0.0850019\tvalid_1's binary_logloss: 0.143491\n",
      "[330]\tvalid_0's binary_logloss: 0.0829341\tvalid_1's binary_logloss: 0.142505\n",
      "[340]\tvalid_0's binary_logloss: 0.0809993\tvalid_1's binary_logloss: 0.141487\n",
      "[350]\tvalid_0's binary_logloss: 0.0790514\tvalid_1's binary_logloss: 0.140376\n",
      "[360]\tvalid_0's binary_logloss: 0.0773814\tvalid_1's binary_logloss: 0.139471\n",
      "[370]\tvalid_0's binary_logloss: 0.0757036\tvalid_1's binary_logloss: 0.138423\n",
      "[380]\tvalid_0's binary_logloss: 0.0741642\tvalid_1's binary_logloss: 0.137519\n",
      "[390]\tvalid_0's binary_logloss: 0.0725658\tvalid_1's binary_logloss: 0.13671\n",
      "[400]\tvalid_0's binary_logloss: 0.0711791\tvalid_1's binary_logloss: 0.136081\n",
      "[410]\tvalid_0's binary_logloss: 0.0698252\tvalid_1's binary_logloss: 0.135343\n",
      "[420]\tvalid_0's binary_logloss: 0.0686159\tvalid_1's binary_logloss: 0.13477\n",
      "[430]\tvalid_0's binary_logloss: 0.0674271\tvalid_1's binary_logloss: 0.134321\n",
      "[440]\tvalid_0's binary_logloss: 0.0663112\tvalid_1's binary_logloss: 0.133698\n",
      "[450]\tvalid_0's binary_logloss: 0.0652003\tvalid_1's binary_logloss: 0.133108\n",
      "[460]\tvalid_0's binary_logloss: 0.0642875\tvalid_1's binary_logloss: 0.132781\n",
      "[470]\tvalid_0's binary_logloss: 0.0633714\tvalid_1's binary_logloss: 0.132364\n",
      "[480]\tvalid_0's binary_logloss: 0.0625077\tvalid_1's binary_logloss: 0.131997\n",
      "[490]\tvalid_0's binary_logloss: 0.0617036\tvalid_1's binary_logloss: 0.131625\n",
      "[500]\tvalid_0's binary_logloss: 0.0608766\tvalid_1's binary_logloss: 0.131107\n",
      "[510]\tvalid_0's binary_logloss: 0.0602016\tvalid_1's binary_logloss: 0.130687\n",
      "[520]\tvalid_0's binary_logloss: 0.0595017\tvalid_1's binary_logloss: 0.130329\n",
      "[530]\tvalid_0's binary_logloss: 0.0589209\tvalid_1's binary_logloss: 0.130086\n",
      "[540]\tvalid_0's binary_logloss: 0.0583273\tvalid_1's binary_logloss: 0.12988\n",
      "[550]\tvalid_0's binary_logloss: 0.0578987\tvalid_1's binary_logloss: 0.129671\n",
      "[560]\tvalid_0's binary_logloss: 0.0573821\tvalid_1's binary_logloss: 0.129472\n",
      "[570]\tvalid_0's binary_logloss: 0.0569517\tvalid_1's binary_logloss: 0.129288\n",
      "[580]\tvalid_0's binary_logloss: 0.0565449\tvalid_1's binary_logloss: 0.129228\n",
      "[590]\tvalid_0's binary_logloss: 0.0561137\tvalid_1's binary_logloss: 0.129024\n",
      "[600]\tvalid_0's binary_logloss: 0.0557219\tvalid_1's binary_logloss: 0.128847\n",
      "[610]\tvalid_0's binary_logloss: 0.0553241\tvalid_1's binary_logloss: 0.128676\n",
      "[620]\tvalid_0's binary_logloss: 0.0550072\tvalid_1's binary_logloss: 0.128542\n",
      "[630]\tvalid_0's binary_logloss: 0.0546755\tvalid_1's binary_logloss: 0.128446\n",
      "[640]\tvalid_0's binary_logloss: 0.0543407\tvalid_1's binary_logloss: 0.128268\n",
      "[650]\tvalid_0's binary_logloss: 0.0540205\tvalid_1's binary_logloss: 0.128154\n",
      "[660]\tvalid_0's binary_logloss: 0.0537046\tvalid_1's binary_logloss: 0.127978\n",
      "[670]\tvalid_0's binary_logloss: 0.0534607\tvalid_1's binary_logloss: 0.127904\n",
      "[680]\tvalid_0's binary_logloss: 0.0531636\tvalid_1's binary_logloss: 0.127782\n",
      "[690]\tvalid_0's binary_logloss: 0.0529777\tvalid_1's binary_logloss: 0.127742\n",
      "[700]\tvalid_0's binary_logloss: 0.0527991\tvalid_1's binary_logloss: 0.127649\n",
      "[710]\tvalid_0's binary_logloss: 0.0526398\tvalid_1's binary_logloss: 0.127566\n",
      "[720]\tvalid_0's binary_logloss: 0.0524829\tvalid_1's binary_logloss: 0.12754\n",
      "[730]\tvalid_0's binary_logloss: 0.0522434\tvalid_1's binary_logloss: 0.127445\n",
      "[740]\tvalid_0's binary_logloss: 0.0520878\tvalid_1's binary_logloss: 0.127356\n",
      "[750]\tvalid_0's binary_logloss: 0.0519547\tvalid_1's binary_logloss: 0.127276\n",
      "[760]\tvalid_0's binary_logloss: 0.0518424\tvalid_1's binary_logloss: 0.127244\n",
      "[770]\tvalid_0's binary_logloss: 0.0517016\tvalid_1's binary_logloss: 0.127162\n",
      "[780]\tvalid_0's binary_logloss: 0.0515726\tvalid_1's binary_logloss: 0.127096\n",
      "[790]\tvalid_0's binary_logloss: 0.0514449\tvalid_1's binary_logloss: 0.127106\n",
      "[800]\tvalid_0's binary_logloss: 0.0512259\tvalid_1's binary_logloss: 0.127064\n",
      "[810]\tvalid_0's binary_logloss: 0.0510892\tvalid_1's binary_logloss: 0.126978\n",
      "[820]\tvalid_0's binary_logloss: 0.0509786\tvalid_1's binary_logloss: 0.126933\n",
      "[830]\tvalid_0's binary_logloss: 0.0508887\tvalid_1's binary_logloss: 0.126938\n",
      "[840]\tvalid_0's binary_logloss: 0.050774\tvalid_1's binary_logloss: 0.126886\n",
      "[850]\tvalid_0's binary_logloss: 0.0506417\tvalid_1's binary_logloss: 0.126823\n",
      "[860]\tvalid_0's binary_logloss: 0.0505282\tvalid_1's binary_logloss: 0.126761\n",
      "[870]\tvalid_0's binary_logloss: 0.0503935\tvalid_1's binary_logloss: 0.126744\n",
      "[880]\tvalid_0's binary_logloss: 0.0502774\tvalid_1's binary_logloss: 0.126725\n",
      "[890]\tvalid_0's binary_logloss: 0.0502115\tvalid_1's binary_logloss: 0.126733\n",
      "[900]\tvalid_0's binary_logloss: 0.0501063\tvalid_1's binary_logloss: 0.126702\n",
      "[910]\tvalid_0's binary_logloss: 0.0500208\tvalid_1's binary_logloss: 0.126666\n",
      "[920]\tvalid_0's binary_logloss: 0.050001\tvalid_1's binary_logloss: 0.126692\n",
      "[930]\tvalid_0's binary_logloss: 0.0499293\tvalid_1's binary_logloss: 0.126686\n",
      "[940]\tvalid_0's binary_logloss: 0.0498259\tvalid_1's binary_logloss: 0.126644\n",
      "[950]\tvalid_0's binary_logloss: 0.0497353\tvalid_1's binary_logloss: 0.12664\n",
      "[960]\tvalid_0's binary_logloss: 0.0496506\tvalid_1's binary_logloss: 0.126606\n",
      "[970]\tvalid_0's binary_logloss: 0.0495821\tvalid_1's binary_logloss: 0.126612\n",
      "[980]\tvalid_0's binary_logloss: 0.0495083\tvalid_1's binary_logloss: 0.126588\n",
      "[990]\tvalid_0's binary_logloss: 0.0494426\tvalid_1's binary_logloss: 0.126566\n",
      "[1000]\tvalid_0's binary_logloss: 0.0494426\tvalid_1's binary_logloss: 0.126566\n",
      "[1010]\tvalid_0's binary_logloss: 0.0494426\tvalid_1's binary_logloss: 0.126566\n",
      "Early stopping, best iteration is:\n",
      "[987]\tvalid_0's binary_logloss: 0.0494488\tvalid_1's binary_logloss: 0.126563\n",
      "0.7452742123687282\n",
      "3\n",
      "Training until validation scores don't improve for 30 rounds.\n",
      "[10]\tvalid_0's binary_logloss: 0.535719\tvalid_1's binary_logloss: 0.540563\n",
      "[20]\tvalid_0's binary_logloss: 0.434989\tvalid_1's binary_logloss: 0.443728\n",
      "[30]\tvalid_0's binary_logloss: 0.365563\tvalid_1's binary_logloss: 0.377718\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[40]\tvalid_0's binary_logloss: 0.316473\tvalid_1's binary_logloss: 0.331386\n",
      "[50]\tvalid_0's binary_logloss: 0.27958\tvalid_1's binary_logloss: 0.297344\n",
      "[60]\tvalid_0's binary_logloss: 0.250155\tvalid_1's binary_logloss: 0.271056\n",
      "[70]\tvalid_0's binary_logloss: 0.22698\tvalid_1's binary_logloss: 0.250638\n",
      "[80]\tvalid_0's binary_logloss: 0.208916\tvalid_1's binary_logloss: 0.23516\n",
      "[90]\tvalid_0's binary_logloss: 0.194405\tvalid_1's binary_logloss: 0.22311\n",
      "[100]\tvalid_0's binary_logloss: 0.18195\tvalid_1's binary_logloss: 0.212909\n",
      "[110]\tvalid_0's binary_logloss: 0.171615\tvalid_1's binary_logloss: 0.204934\n",
      "[120]\tvalid_0's binary_logloss: 0.162621\tvalid_1's binary_logloss: 0.198312\n",
      "[130]\tvalid_0's binary_logloss: 0.154743\tvalid_1's binary_logloss: 0.192804\n",
      "[140]\tvalid_0's binary_logloss: 0.147647\tvalid_1's binary_logloss: 0.187377\n",
      "[150]\tvalid_0's binary_logloss: 0.141229\tvalid_1's binary_logloss: 0.182767\n",
      "[160]\tvalid_0's binary_logloss: 0.135634\tvalid_1's binary_logloss: 0.178876\n",
      "[170]\tvalid_0's binary_logloss: 0.130347\tvalid_1's binary_logloss: 0.175402\n",
      "[180]\tvalid_0's binary_logloss: 0.125419\tvalid_1's binary_logloss: 0.172389\n",
      "[190]\tvalid_0's binary_logloss: 0.120871\tvalid_1's binary_logloss: 0.169321\n",
      "[200]\tvalid_0's binary_logloss: 0.116741\tvalid_1's binary_logloss: 0.166647\n",
      "[210]\tvalid_0's binary_logloss: 0.112707\tvalid_1's binary_logloss: 0.163993\n",
      "[220]\tvalid_0's binary_logloss: 0.109035\tvalid_1's binary_logloss: 0.161758\n",
      "[230]\tvalid_0's binary_logloss: 0.105597\tvalid_1's binary_logloss: 0.159652\n",
      "[240]\tvalid_0's binary_logloss: 0.102384\tvalid_1's binary_logloss: 0.157733\n",
      "[250]\tvalid_0's binary_logloss: 0.0992971\tvalid_1's binary_logloss: 0.155858\n",
      "[260]\tvalid_0's binary_logloss: 0.0963575\tvalid_1's binary_logloss: 0.154203\n",
      "[270]\tvalid_0's binary_logloss: 0.0936586\tvalid_1's binary_logloss: 0.15293\n",
      "[280]\tvalid_0's binary_logloss: 0.0909846\tvalid_1's binary_logloss: 0.151421\n",
      "[290]\tvalid_0's binary_logloss: 0.0884037\tvalid_1's binary_logloss: 0.149942\n",
      "[300]\tvalid_0's binary_logloss: 0.0859638\tvalid_1's binary_logloss: 0.148609\n",
      "[310]\tvalid_0's binary_logloss: 0.083833\tvalid_1's binary_logloss: 0.147437\n",
      "[320]\tvalid_0's binary_logloss: 0.0815795\tvalid_1's binary_logloss: 0.146082\n",
      "[330]\tvalid_0's binary_logloss: 0.0795458\tvalid_1's binary_logloss: 0.14487\n",
      "[340]\tvalid_0's binary_logloss: 0.0776791\tvalid_1's binary_logloss: 0.143919\n",
      "[350]\tvalid_0's binary_logloss: 0.0758594\tvalid_1's binary_logloss: 0.143012\n",
      "[360]\tvalid_0's binary_logloss: 0.0741812\tvalid_1's binary_logloss: 0.142134\n",
      "[370]\tvalid_0's binary_logloss: 0.0724951\tvalid_1's binary_logloss: 0.141235\n",
      "[380]\tvalid_0's binary_logloss: 0.0710384\tvalid_1's binary_logloss: 0.140491\n",
      "[390]\tvalid_0's binary_logloss: 0.0695424\tvalid_1's binary_logloss: 0.139702\n",
      "[400]\tvalid_0's binary_logloss: 0.0682131\tvalid_1's binary_logloss: 0.139067\n",
      "[410]\tvalid_0's binary_logloss: 0.0669509\tvalid_1's binary_logloss: 0.13848\n",
      "[420]\tvalid_0's binary_logloss: 0.0656975\tvalid_1's binary_logloss: 0.137877\n",
      "[430]\tvalid_0's binary_logloss: 0.0645588\tvalid_1's binary_logloss: 0.137319\n",
      "[440]\tvalid_0's binary_logloss: 0.0635368\tvalid_1's binary_logloss: 0.136784\n",
      "[450]\tvalid_0's binary_logloss: 0.0625277\tvalid_1's binary_logloss: 0.136388\n",
      "[460]\tvalid_0's binary_logloss: 0.0616054\tvalid_1's binary_logloss: 0.135974\n",
      "[470]\tvalid_0's binary_logloss: 0.0608351\tvalid_1's binary_logloss: 0.13559\n",
      "[480]\tvalid_0's binary_logloss: 0.0600588\tvalid_1's binary_logloss: 0.135238\n",
      "[490]\tvalid_0's binary_logloss: 0.059297\tvalid_1's binary_logloss: 0.134953\n",
      "[500]\tvalid_0's binary_logloss: 0.0586014\tvalid_1's binary_logloss: 0.134684\n",
      "[510]\tvalid_0's binary_logloss: 0.0579668\tvalid_1's binary_logloss: 0.134391\n",
      "[520]\tvalid_0's binary_logloss: 0.0573639\tvalid_1's binary_logloss: 0.134131\n",
      "[530]\tvalid_0's binary_logloss: 0.0567767\tvalid_1's binary_logloss: 0.133964\n",
      "[540]\tvalid_0's binary_logloss: 0.0562982\tvalid_1's binary_logloss: 0.133776\n",
      "[550]\tvalid_0's binary_logloss: 0.0557418\tvalid_1's binary_logloss: 0.133492\n",
      "[560]\tvalid_0's binary_logloss: 0.055305\tvalid_1's binary_logloss: 0.133349\n",
      "[570]\tvalid_0's binary_logloss: 0.0548435\tvalid_1's binary_logloss: 0.133153\n",
      "[580]\tvalid_0's binary_logloss: 0.0545004\tvalid_1's binary_logloss: 0.132968\n",
      "[590]\tvalid_0's binary_logloss: 0.0540718\tvalid_1's binary_logloss: 0.132734\n",
      "[600]\tvalid_0's binary_logloss: 0.0537453\tvalid_1's binary_logloss: 0.132646\n",
      "[610]\tvalid_0's binary_logloss: 0.0533697\tvalid_1's binary_logloss: 0.132538\n",
      "[620]\tvalid_0's binary_logloss: 0.0530512\tvalid_1's binary_logloss: 0.132416\n",
      "[630]\tvalid_0's binary_logloss: 0.0527806\tvalid_1's binary_logloss: 0.132307\n",
      "[640]\tvalid_0's binary_logloss: 0.0524872\tvalid_1's binary_logloss: 0.132196\n",
      "[650]\tvalid_0's binary_logloss: 0.0522461\tvalid_1's binary_logloss: 0.132119\n",
      "[660]\tvalid_0's binary_logloss: 0.0520548\tvalid_1's binary_logloss: 0.132074\n",
      "[670]\tvalid_0's binary_logloss: 0.0518063\tvalid_1's binary_logloss: 0.131989\n",
      "[680]\tvalid_0's binary_logloss: 0.0514996\tvalid_1's binary_logloss: 0.131841\n",
      "[690]\tvalid_0's binary_logloss: 0.0512\tvalid_1's binary_logloss: 0.131667\n",
      "[700]\tvalid_0's binary_logloss: 0.0509946\tvalid_1's binary_logloss: 0.131585\n",
      "[710]\tvalid_0's binary_logloss: 0.0507921\tvalid_1's binary_logloss: 0.131567\n",
      "[720]\tvalid_0's binary_logloss: 0.0506461\tvalid_1's binary_logloss: 0.131539\n",
      "[730]\tvalid_0's binary_logloss: 0.0505379\tvalid_1's binary_logloss: 0.131528\n",
      "[740]\tvalid_0's binary_logloss: 0.0503632\tvalid_1's binary_logloss: 0.131489\n",
      "[750]\tvalid_0's binary_logloss: 0.0501356\tvalid_1's binary_logloss: 0.131387\n",
      "[760]\tvalid_0's binary_logloss: 0.0499126\tvalid_1's binary_logloss: 0.131319\n",
      "[770]\tvalid_0's binary_logloss: 0.0498091\tvalid_1's binary_logloss: 0.131263\n",
      "[780]\tvalid_0's binary_logloss: 0.0496073\tvalid_1's binary_logloss: 0.131174\n",
      "[790]\tvalid_0's binary_logloss: 0.0495126\tvalid_1's binary_logloss: 0.131168\n",
      "[800]\tvalid_0's binary_logloss: 0.0494358\tvalid_1's binary_logloss: 0.131141\n",
      "[810]\tvalid_0's binary_logloss: 0.0493017\tvalid_1's binary_logloss: 0.13109\n",
      "[820]\tvalid_0's binary_logloss: 0.0491693\tvalid_1's binary_logloss: 0.131043\n",
      "[830]\tvalid_0's binary_logloss: 0.0490702\tvalid_1's binary_logloss: 0.131071\n",
      "[840]\tvalid_0's binary_logloss: 0.0490191\tvalid_1's binary_logloss: 0.131082\n",
      "[850]\tvalid_0's binary_logloss: 0.0490191\tvalid_1's binary_logloss: 0.131082\n",
      "Early stopping, best iteration is:\n",
      "[820]\tvalid_0's binary_logloss: 0.0491693\tvalid_1's binary_logloss: 0.131043\n",
      "0.7298716452742122\n",
      "4\n",
      "Training until validation scores don't improve for 30 rounds.\n",
      "[10]\tvalid_0's binary_logloss: 0.534835\tvalid_1's binary_logloss: 0.539653\n",
      "[20]\tvalid_0's binary_logloss: 0.434387\tvalid_1's binary_logloss: 0.44255\n",
      "[30]\tvalid_0's binary_logloss: 0.364256\tvalid_1's binary_logloss: 0.374875\n",
      "[40]\tvalid_0's binary_logloss: 0.313626\tvalid_1's binary_logloss: 0.326884\n",
      "[50]\tvalid_0's binary_logloss: 0.276778\tvalid_1's binary_logloss: 0.292161\n",
      "[60]\tvalid_0's binary_logloss: 0.247644\tvalid_1's binary_logloss: 0.265526\n",
      "[70]\tvalid_0's binary_logloss: 0.224927\tvalid_1's binary_logloss: 0.245227\n",
      "[80]\tvalid_0's binary_logloss: 0.20681\tvalid_1's binary_logloss: 0.228974\n",
      "[90]\tvalid_0's binary_logloss: 0.192353\tvalid_1's binary_logloss: 0.216837\n",
      "[100]\tvalid_0's binary_logloss: 0.18005\tvalid_1's binary_logloss: 0.206896\n",
      "[110]\tvalid_0's binary_logloss: 0.170062\tvalid_1's binary_logloss: 0.198899\n",
      "[120]\tvalid_0's binary_logloss: 0.161301\tvalid_1's binary_logloss: 0.192452\n",
      "[130]\tvalid_0's binary_logloss: 0.153305\tvalid_1's binary_logloss: 0.186746\n",
      "[140]\tvalid_0's binary_logloss: 0.146411\tvalid_1's binary_logloss: 0.181758\n",
      "[150]\tvalid_0's binary_logloss: 0.139906\tvalid_1's binary_logloss: 0.177273\n",
      "[160]\tvalid_0's binary_logloss: 0.133951\tvalid_1's binary_logloss: 0.173299\n",
      "[170]\tvalid_0's binary_logloss: 0.128802\tvalid_1's binary_logloss: 0.169972\n",
      "[180]\tvalid_0's binary_logloss: 0.123834\tvalid_1's binary_logloss: 0.167006\n",
      "[190]\tvalid_0's binary_logloss: 0.119421\tvalid_1's binary_logloss: 0.164514\n",
      "[200]\tvalid_0's binary_logloss: 0.115236\tvalid_1's binary_logloss: 0.16211\n",
      "[210]\tvalid_0's binary_logloss: 0.111427\tvalid_1's binary_logloss: 0.159994\n",
      "[220]\tvalid_0's binary_logloss: 0.107636\tvalid_1's binary_logloss: 0.158158\n",
      "[230]\tvalid_0's binary_logloss: 0.104227\tvalid_1's binary_logloss: 0.156359\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[240]\tvalid_0's binary_logloss: 0.101131\tvalid_1's binary_logloss: 0.154777\n",
      "[250]\tvalid_0's binary_logloss: 0.0981556\tvalid_1's binary_logloss: 0.15315\n",
      "[260]\tvalid_0's binary_logloss: 0.095287\tvalid_1's binary_logloss: 0.151646\n",
      "[270]\tvalid_0's binary_logloss: 0.0925143\tvalid_1's binary_logloss: 0.150282\n",
      "[280]\tvalid_0's binary_logloss: 0.0899941\tvalid_1's binary_logloss: 0.149172\n",
      "[290]\tvalid_0's binary_logloss: 0.0874351\tvalid_1's binary_logloss: 0.147857\n",
      "[300]\tvalid_0's binary_logloss: 0.0852004\tvalid_1's binary_logloss: 0.14707\n",
      "[310]\tvalid_0's binary_logloss: 0.0829812\tvalid_1's binary_logloss: 0.146053\n",
      "[320]\tvalid_0's binary_logloss: 0.0808595\tvalid_1's binary_logloss: 0.145246\n",
      "[330]\tvalid_0's binary_logloss: 0.0787382\tvalid_1's binary_logloss: 0.144267\n",
      "[340]\tvalid_0's binary_logloss: 0.0766852\tvalid_1's binary_logloss: 0.143295\n",
      "[350]\tvalid_0's binary_logloss: 0.0749009\tvalid_1's binary_logloss: 0.142683\n",
      "[360]\tvalid_0's binary_logloss: 0.0730735\tvalid_1's binary_logloss: 0.141958\n",
      "[370]\tvalid_0's binary_logloss: 0.0715251\tvalid_1's binary_logloss: 0.14146\n",
      "[380]\tvalid_0's binary_logloss: 0.0699716\tvalid_1's binary_logloss: 0.140911\n",
      "[390]\tvalid_0's binary_logloss: 0.068468\tvalid_1's binary_logloss: 0.140216\n",
      "[400]\tvalid_0's binary_logloss: 0.067186\tvalid_1's binary_logloss: 0.139799\n",
      "[410]\tvalid_0's binary_logloss: 0.065985\tvalid_1's binary_logloss: 0.139323\n",
      "[420]\tvalid_0's binary_logloss: 0.0649079\tvalid_1's binary_logloss: 0.139021\n",
      "[430]\tvalid_0's binary_logloss: 0.0637836\tvalid_1's binary_logloss: 0.138592\n",
      "[440]\tvalid_0's binary_logloss: 0.0626513\tvalid_1's binary_logloss: 0.138086\n",
      "[450]\tvalid_0's binary_logloss: 0.061704\tvalid_1's binary_logloss: 0.137835\n",
      "[460]\tvalid_0's binary_logloss: 0.0607768\tvalid_1's binary_logloss: 0.137568\n",
      "[470]\tvalid_0's binary_logloss: 0.0599592\tvalid_1's binary_logloss: 0.137367\n",
      "[480]\tvalid_0's binary_logloss: 0.059138\tvalid_1's binary_logloss: 0.137129\n",
      "[490]\tvalid_0's binary_logloss: 0.0583228\tvalid_1's binary_logloss: 0.136893\n",
      "[500]\tvalid_0's binary_logloss: 0.0576567\tvalid_1's binary_logloss: 0.136685\n",
      "[510]\tvalid_0's binary_logloss: 0.0570037\tvalid_1's binary_logloss: 0.136572\n",
      "[520]\tvalid_0's binary_logloss: 0.0564408\tvalid_1's binary_logloss: 0.136474\n",
      "[530]\tvalid_0's binary_logloss: 0.0558784\tvalid_1's binary_logloss: 0.136268\n",
      "[540]\tvalid_0's binary_logloss: 0.0553056\tvalid_1's binary_logloss: 0.136101\n",
      "[550]\tvalid_0's binary_logloss: 0.0548896\tvalid_1's binary_logloss: 0.136038\n",
      "[560]\tvalid_0's binary_logloss: 0.0544201\tvalid_1's binary_logloss: 0.135931\n",
      "[570]\tvalid_0's binary_logloss: 0.0540288\tvalid_1's binary_logloss: 0.135806\n",
      "[580]\tvalid_0's binary_logloss: 0.0536598\tvalid_1's binary_logloss: 0.135812\n",
      "[590]\tvalid_0's binary_logloss: 0.0533072\tvalid_1's binary_logloss: 0.135784\n",
      "[600]\tvalid_0's binary_logloss: 0.0529997\tvalid_1's binary_logloss: 0.135707\n",
      "[610]\tvalid_0's binary_logloss: 0.0526879\tvalid_1's binary_logloss: 0.135588\n",
      "[620]\tvalid_0's binary_logloss: 0.0524319\tvalid_1's binary_logloss: 0.135533\n",
      "[630]\tvalid_0's binary_logloss: 0.0521264\tvalid_1's binary_logloss: 0.135443\n",
      "[640]\tvalid_0's binary_logloss: 0.0518999\tvalid_1's binary_logloss: 0.135409\n",
      "[650]\tvalid_0's binary_logloss: 0.051583\tvalid_1's binary_logloss: 0.135292\n",
      "[660]\tvalid_0's binary_logloss: 0.0513673\tvalid_1's binary_logloss: 0.135221\n",
      "[670]\tvalid_0's binary_logloss: 0.051041\tvalid_1's binary_logloss: 0.135152\n",
      "[680]\tvalid_0's binary_logloss: 0.0508123\tvalid_1's binary_logloss: 0.135093\n",
      "[690]\tvalid_0's binary_logloss: 0.0505398\tvalid_1's binary_logloss: 0.135067\n",
      "[700]\tvalid_0's binary_logloss: 0.0503607\tvalid_1's binary_logloss: 0.135038\n",
      "[710]\tvalid_0's binary_logloss: 0.0501591\tvalid_1's binary_logloss: 0.134992\n",
      "[720]\tvalid_0's binary_logloss: 0.0500621\tvalid_1's binary_logloss: 0.134988\n",
      "[730]\tvalid_0's binary_logloss: 0.0498366\tvalid_1's binary_logloss: 0.134935\n",
      "[740]\tvalid_0's binary_logloss: 0.0496609\tvalid_1's binary_logloss: 0.134877\n",
      "[750]\tvalid_0's binary_logloss: 0.0494698\tvalid_1's binary_logloss: 0.134868\n",
      "[760]\tvalid_0's binary_logloss: 0.0493257\tvalid_1's binary_logloss: 0.134883\n",
      "[770]\tvalid_0's binary_logloss: 0.0491604\tvalid_1's binary_logloss: 0.134899\n",
      "[780]\tvalid_0's binary_logloss: 0.0490326\tvalid_1's binary_logloss: 0.134895\n",
      "Early stopping, best iteration is:\n",
      "[752]\tvalid_0's binary_logloss: 0.0494327\tvalid_1's binary_logloss: 0.134848\n",
      "0.7147024504084014\n",
      "[0.738389731621937, 0.7409568261376898, 0.7452742123687282, 0.7298716452742122, 0.7147024504084014]\n",
      "0.7338389731621936\n",
      "123\n",
      "0.7452742123687282\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "from base_helper import *\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "sub = sub_csv\n",
    "\n",
    "train = df_train\n",
    "label = df_train.Tag.values\n",
    "train.drop('Tag', axis=1, inplace=True)\n",
    "test = df_test\n",
    "\n",
    "print(test.shape)\n",
    "\n",
    "train = train.values\n",
    "test = test.values\n",
    "\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "skf = StratifiedKFold(n_splits=5, random_state=2018, shuffle=True)\n",
    "best_score = []\n",
    "\n",
    "oof_preds = np.zeros(train.shape[0])\n",
    "sub_preds = np.zeros(test.shape[0])\n",
    "res, weights = pd.DataFrame(), []\n",
    "for index, (train_index, test_index) in enumerate(skf.split(train, label)):\n",
    "    print(index)\n",
    "    lgb_model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=62, reg_alpha=3, reg_lambda=5, max_depth=-1,\n",
    "                                   n_estimators=5000, objective='binary', subsample=0.95, colsample_bytree=0.818,\n",
    "                                   subsample_freq=1, learning_rate=0.03,\n",
    "                                   class_weight='balanced',\n",
    "                                   random_state=1000 + index, n_jobs=4, min_child_weight=8, min_child_samples=8,\n",
    "                                   min_split_gain=0.2)\n",
    "    lgb_model.fit(train[train_index], label[train_index], verbose=10,\n",
    "                  eval_metric=\"binary_logloss\",\n",
    "                  eval_set=[(train[train_index], label[train_index]),\n",
    "                            (train[test_index], label[test_index])], early_stopping_rounds=30)\n",
    "    oof_preds[test_index] = lgb_model.predict_proba(train[test_index], num_iteration=lgb_model.best_iteration_)[:, 1]\n",
    "    m = tpr_weight_funtion(y_predict=oof_preds[test_index], y_true=label[test_index])\n",
    "    print(m)\n",
    "    # if m < 0.82:\n",
    "    #     continue\n",
    "    test_pred = lgb_model.predict_proba(test, num_iteration=lgb_model.best_iteration_)[:, 1]\n",
    "    weights.append(m)\n",
    "    res[index] = test_pred\n",
    "    # sub_preds += test_pred / 5\n",
    "    # print('test mean:', test_pred.mean())\n",
    "    # predict_result['predicted_score'] = predict_result['predicted_score'] + test_pred\n",
    "\n",
    "print(weights)\n",
    "res.columns = [i for i in range(res.shape[1])]\n",
    "blending_model(res, weights, \"baseline12_\", sub)\n",
    "print(123)\n",
    "index = weights.index(max(weights))\n",
    "sub['Tag'] = res[index]\n",
    "print(max(weights))\n",
    "sub.to_csv('baseline10_%s.csv' % str(weights[index]), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
